package spark.mllib

import org.apache.spark.{SparkConf, SparkContext}

/**
  * Created by lenovo on 2017/5/5.
  */
object License1 {

  def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setAppName("lw").setMaster("local[5]")
    val sc = new SparkContext(conf)
    val rdd1 = sc.parallelize(1 to 9, 3)
    val rdd2 = rdd1.map(x => x * 2)
    //    rdd2.collect
    rdd2.collect.foreach(print)
    val rdd3 = rdd2.filter(x => x > 10)
    rdd3.collect.foreach(print)
    val rdd4 = rdd3.flatMap(x => x to 20)
    rdd4.collect.foreach(print)
    val a = sc.parallelize(1 to 10000, 3)
    val res24 = a.sample(false, 0.1, 0).count
    println(res24)
    val rdd8 = rdd1.union(rdd3)
    rdd8.collect.foreach(print)
    val rdd9 = rdd8.intersection(rdd1)
    rdd9.collect.foreach(print)
    val rdd10 = rdd8.union(rdd9).distinct
    rdd10.collect.foreach(print)
    val rdd0 = sc.parallelize(Array((1, 1), (1, 2), (1, 3), (2, 1), (2, 2), (2, 3)), 3)
    val rdd11 = rdd0.groupByKey()
    rdd11.collect.foreach(print)
    val rdd12 = rdd0.reduceByKey((x, y) => x + y)
    rdd12.collect.foreach(print)
    //    val z1 = sc.parallelize(List(1,2,3,4,5,6), 2)
    //    z.aggregate(0,12,3)
    //    z1.aggreate(0)(math.max(_, _), _ + _)

    val z = sc.parallelize(List((1, 3), (1, 2), (1, 4), (2, 3)))
    z.aggregateByKey(0)(math.max(_, _), _ + _).collect.foreach(print)

//    val data = Array((1, 1.0), (1, 2.0), (1, 3.0), (2, 4.0), (2, 5.0), (2, 6.0))
//    val rdd = sc.parallelize(data, 2)
//    val combine1 = rdd.combineByKey(createCombiner = (v: Double) => (v: Double, 1),
//      mergeValue = (c: (Double, Int), v: Double) => (c._1 + v, c._2 + 1),
//      mergeCombiners = (c1: (Double, Int), c2: (Double, Int)) => (c1._1 + c2._1, c1._2 + c2._2),
//      numPartitions = 2)
//    combine1.collect.foreach(print)
//
//    val rdd14 = rdd0.sortByKey()
//    rdd14.collect.foreach(print)
//
//    val rdd15 = rdd0.join(rdd0)
//    rdd15.collect.foreach(print)
//
//    val rdd16 = rdd0.cogroup(rdd0)
//    rdd16.collect.foreach(print)
//
//    val rdd17 = rdd1.cartesian(rdd3)
//    rdd17.collect.foreach(print)
  }

}
